Evaluate Models
Author: Aaron S Coyner, PhD
Last update: 2021-09-27
Setup
library(tidyverse)
library(janitor)
library(tools)
library(caret)
library(ROCR)
library(caret)compute_auroc <- function(predictions, labels, title) {
pred <- prediction(predictions, labels)
auroc <- performance(pred, measure = 'auc')
auroc <- auroc@y.values[[1]]
perf <- performance(pred, 'tpr', 'fpr')
plot(perf,
main = title,
xlim=c(0,1),
ylim=c(0,1),
colorize = FALSE)
abline(a = 0,
b = 1,
col = 'red',
lty = 2)
text(0.8,
0.25,
paste('AUROC:', sprintf('%.3f', round(auroc, 3))))
}compute_aupr <- function(predictions, labels, title) {
n_black <- length(labels[labels == 1])
n_total <- length(labels)
pred <- prediction(predictions, labels)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
perf <- performance(pred, 'prec', 'rec')
plot(perf,
main = title,
xlim=c(0,1),
ylim=c(0,1),
colorize = FALSE)
abline(a = n_black/n_total,
b = 0,
col = 'red',
lty = 2)
text(0.2,
0.2,
paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))))
text(0.8,
0.2,
paste('AUPR:', sprintf('%.3f', round(aupr, 3))))
}confusion_matrix <- function(df, column, threshold=0.5) {
preds <- as.factor(if_else(df[column] >= threshold, 1, 0))
confusionMatrix(preds, df$label, positive = '1')
}Load data
irop_data <- read_csv('/Volumes/External/irop_data/irop_07092020.csv') %>%
clean_names() %>%
distinct(subject_id, .keep_all = TRUE) %>%
select(subject_id, race)
── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
.default = col_character(),
subjectVisitID = col_double(),
birthWeight = col_double(),
gestationalAgeWeeks = col_double(),
gestationalAgeDays = col_double(),
`PMA Weeks` = col_double(),
`PMA Days` = col_double(),
PMARaw = col_double(),
`Session Followup` = col_double(),
`Golden Reading Stage` = col_double(),
bloodTaken = col_double(),
salivaTaken = col_double()
)
ℹ Use `spec()` for the full column specifications.Warning: 5380 parsing failures.
row col expected actual file
1645 subjectVisitID a double NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 PMA Weeks a double NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 PMA Days a double NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 PMARaw a double NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 Session Followup a double NULL '/Volumes/External/irop_data/irop_07092020.csv'
.... ................ ........ ...... ...............................................
See problems(...) for more details.test_data <- read_csv('./out/datasets/test_data.csv') %>%
select(subject_id, image_id)
── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
subject_id = col_character(),
race = col_character(),
variable = col_character(),
value = col_character(),
image_id = col_double(),
fundus_location = col_character(),
segmentation_location = col_character()
)image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
select(img_loc, image_id, label, retcam = probability) %>%
bind_cols(select(read_csv('./out/probabilities/retcam_filtered_0_random.csv', col_types = cols()), retcam_random = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_random.csv', col_types = cols()), filter_0_random = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0.csv', col_types = cols()), filter_0 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_binarized.csv', col_types = cols()), filter_0_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_skeletonized.csv', col_types = cols()), filter_0_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50.csv', col_types = cols()), filter_50 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_binarized.csv', col_types = cols()), filter_50_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_skeletonized.csv', col_types = cols()), filter_50_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100.csv', col_types = cols()), filter_100 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_binarized.csv', col_types = cols()), filter_100_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_skeletonized.csv', col_types = cols()), filter_100_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150.csv', col_types = cols()), filter_150 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_binarized.csv', col_types = cols()), filter_150_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_skeletonized.csv', col_types = cols()), filter_150_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200.csv', col_types = cols()), filter_200 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_binarized.csv', col_types = cols()), filter_200_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_skeletonized.csv', col_types = cols()), filter_200_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210.csv', col_types = cols()), filter_210 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_binarized.csv', col_types = cols()), filter_210_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_skeletonized.csv', col_types = cols()), filter_210_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220.csv', col_types = cols()), filter_220 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_binarized.csv', col_types = cols()), filter_220_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_skeletonized.csv', col_types = cols()), filter_220_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230.csv', col_types = cols()), filter_230 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_binarized.csv', col_types = cols()), filter_230_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_skeletonized.csv', col_types = cols()), filter_230_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240.csv', col_types = cols()), filter_240 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_binarized.csv', col_types = cols()), filter_240_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_skeletonized.csv', col_types = cols()), filter_240_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250.csv', col_types = cols()), filter_250 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_binarized.csv', col_types = cols()), filter_250_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_skeletonized.csv', col_types = cols()), filter_250_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257.csv', col_types = cols()), filter_257 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_binarized.csv', col_types = cols()), filter_257_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_skeletonized.csv', col_types = cols()), filter_257_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10.csv', col_types = cols()), filter_10 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_binarized.csv', col_types = cols()), filter_10_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_skeletonized.csv', col_types = cols()), filter_10_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150.csv', col_types = cols()), filter_75 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_binarized.csv', col_types = cols()), filter_75_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_skeletonized.csv', col_types = cols()), filter_75_skeletonized = probability)) %>%
inner_join(test_data, by = 'image_id') %>%
select(subject_id, everything(), -img_loc)
subject_level <- image_level %>%
group_by(subject_id) %>%
mutate(across(c(-image_id, -label), median)) %>%
ungroup() %>%
distinct(subject_id, .keep_all = TRUE)Image-level Analysis
RetCam Images
compute_aupr(image_level$retcam, image_level$label, 'PR: Raw RetCam Images')
compute_auroc(image_level$retcam, image_level$label, 'ROC: Raw RetCam Images')
confusion_matrix(image_level, 'retcam')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 819 10
1 8 513
Accuracy : 0.9867
95% CI : (0.979, 0.9921)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.9719
Mcnemar's Test P-Value : 0.8137
Sensitivity : 0.9809
Specificity : 0.9903
Pos Pred Value : 0.9846
Neg Pred Value : 0.9879
Prevalence : 0.3874
Detection Rate : 0.3800
Detection Prevalence : 0.3859
Balanced Accuracy : 0.9856
'Positive' Class : 1
compute_aupr(image_level$retcam_random, image_level$label, 'PR: Raw RetCam Images - Shuffled Labels')
compute_auroc(image_level$retcam_random, image_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')
confusion_matrix(image_level, 'retcam_random')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 444 113
1 383 410
Accuracy : 0.6326
95% CI : (0.6062, 0.6584)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.06902
Kappa : 0.293
Mcnemar's Test P-Value : < 2e-16
Sensitivity : 0.7839
Specificity : 0.5369
Pos Pred Value : 0.5170
Neg Pred Value : 0.7971
Prevalence : 0.3874
Detection Rate : 0.3037
Detection Prevalence : 0.5874
Balanced Accuracy : 0.6604
'Positive' Class : 1
Segmentations: zero all pixels < 0
compute_aupr(image_level$filter_0, image_level$label, 'PR: Segmentations: zero all pixels < 0')
compute_auroc(image_level$filter_0, image_level$label, 'ROC: Segmentations: zero all pixels < 0')
confusion_matrix(image_level, 'filter_0')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 758 74
1 69 449
Accuracy : 0.8941
95% CI : (0.8764, 0.91)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.7764
Mcnemar's Test P-Value : 0.738
Sensitivity : 0.8585
Specificity : 0.9166
Pos Pred Value : 0.8668
Neg Pred Value : 0.9111
Prevalence : 0.3874
Detection Rate : 0.3326
Detection Prevalence : 0.3837
Balanced Accuracy : 0.8875
'Positive' Class : 1
compute_aupr(image_level$filter_0_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')
compute_auroc(image_level$filter_0_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')
confusion_matrix(image_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 733 31
1 94 492
Accuracy : 0.9074
95% CI : (0.8907, 0.9223)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.8091
Mcnemar's Test P-Value : 2.932e-08
Sensitivity : 0.9407
Specificity : 0.8863
Pos Pred Value : 0.8396
Neg Pred Value : 0.9594
Prevalence : 0.3874
Detection Rate : 0.3644
Detection Prevalence : 0.4341
Balanced Accuracy : 0.9135
'Positive' Class : 1
compute_aupr(image_level$filter_0_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')
compute_auroc(image_level$filter_0_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')
confusion_matrix(image_level, 'filter_0_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 691 38
1 136 485
Accuracy : 0.8711
95% CI : (0.8521, 0.8885)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.7375
Mcnemar's Test P-Value : 1.93e-13
Sensitivity : 0.9273
Specificity : 0.8356
Pos Pred Value : 0.7810
Neg Pred Value : 0.9479
Prevalence : 0.3874
Detection Rate : 0.3593
Detection Prevalence : 0.4600
Balanced Accuracy : 0.8814
'Positive' Class : 1
compute_aupr(image_level$filter_0_random, image_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')
compute_auroc(image_level$filter_0_random, image_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')
confusion_matrix(image_level, 'filter_0_random')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 703 418
1 124 105
Accuracy : 0.5985
95% CI : (0.5718, 0.6248)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.8619
Kappa : 0.0567
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.20076
Specificity : 0.85006
Pos Pred Value : 0.45852
Neg Pred Value : 0.62712
Prevalence : 0.38741
Detection Rate : 0.07778
Detection Prevalence : 0.16963
Balanced Accuracy : 0.52541
'Positive' Class : 1
Segmentations: zero all pixels < 50
compute_aupr(image_level$filter_50, image_level$label, 'PR: Segmentations: zero all pixels < 50')
compute_auroc(image_level$filter_50, image_level$label, 'ROC: Segmentations: zero all pixels < 50')
confusion_matrix(image_level, 'filter_50')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 588 63
1 239 460
Accuracy : 0.7763
95% CI : (0.7531, 0.7983)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5561
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.8795
Specificity : 0.7110
Pos Pred Value : 0.6581
Neg Pred Value : 0.9032
Prevalence : 0.3874
Detection Rate : 0.3407
Detection Prevalence : 0.5178
Balanced Accuracy : 0.7953
'Positive' Class : 1
compute_aupr(image_level$filter_50_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')
compute_auroc(image_level$filter_50_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')
confusion_matrix(image_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 733 31
1 94 492
Accuracy : 0.9074
95% CI : (0.8907, 0.9223)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.8091
Mcnemar's Test P-Value : 2.932e-08
Sensitivity : 0.9407
Specificity : 0.8863
Pos Pred Value : 0.8396
Neg Pred Value : 0.9594
Prevalence : 0.3874
Detection Rate : 0.3644
Detection Prevalence : 0.4341
Balanced Accuracy : 0.9135
'Positive' Class : 1
compute_aupr(image_level$filter_50_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')
compute_auroc(image_level$filter_50_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')
confusion_matrix(image_level, 'filter_50_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 408 20
1 419 503
Accuracy : 0.6748
95% CI : (0.6491, 0.6998)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.202e-06
Kappa : 0.3991
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.9618
Specificity : 0.4933
Pos Pred Value : 0.5456
Neg Pred Value : 0.9533
Prevalence : 0.3874
Detection Rate : 0.3726
Detection Prevalence : 0.6830
Balanced Accuracy : 0.7276
'Positive' Class : 1
Segmentations: zero all pixels < 100
compute_aupr(image_level$filter_100, image_level$label, 'PR: Segmentations: zero all pixels < 100')
compute_auroc(image_level$filter_100, image_level$label, 'ROC: Segmentations: zero all pixels < 100')
confusion_matrix(image_level, 'filter_100')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 736 210
1 91 313
Accuracy : 0.777
95% CI : (0.7539, 0.799)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5098
Mcnemar's Test P-Value : 1.036e-11
Sensitivity : 0.5985
Specificity : 0.8900
Pos Pred Value : 0.7748
Neg Pred Value : 0.7780
Prevalence : 0.3874
Detection Rate : 0.2319
Detection Prevalence : 0.2993
Balanced Accuracy : 0.7442
'Positive' Class : 1
compute_aupr(image_level$filter_100_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')
compute_auroc(image_level$filter_100_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')
confusion_matrix(image_level, 'filter_100_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 669 142
1 158 381
Accuracy : 0.7778
95% CI : (0.7546, 0.7997)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.5344
Mcnemar's Test P-Value : 0.3865
Sensitivity : 0.7285
Specificity : 0.8089
Pos Pred Value : 0.7069
Neg Pred Value : 0.8249
Prevalence : 0.3874
Detection Rate : 0.2822
Detection Prevalence : 0.3993
Balanced Accuracy : 0.7687
'Positive' Class : 1
compute_aupr(image_level$filter_100_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')
compute_auroc(image_level$filter_100_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')
confusion_matrix(image_level, 'filter_100_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 632 144
1 195 379
Accuracy : 0.7489
95% CI : (0.7249, 0.7718)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.4803
Mcnemar's Test P-Value : 0.006615
Sensitivity : 0.7247
Specificity : 0.7642
Pos Pred Value : 0.6603
Neg Pred Value : 0.8144
Prevalence : 0.3874
Detection Rate : 0.2807
Detection Prevalence : 0.4252
Balanced Accuracy : 0.7444
'Positive' Class : 1
Segmentations: zero all pixels < 150
compute_aupr(image_level$filter_150, image_level$label, 'PR: Segmentations: zero all pixels < 150')
compute_auroc(image_level$filter_150, image_level$label, 'ROC: Segmentations: zero all pixels < 150')
confusion_matrix(image_level, 'filter_150')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 625 118
1 202 405
Accuracy : 0.763
95% CI : (0.7393, 0.7854)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5149
Mcnemar's Test P-Value : 3.487e-06
Sensitivity : 0.7744
Specificity : 0.7557
Pos Pred Value : 0.6672
Neg Pred Value : 0.8412
Prevalence : 0.3874
Detection Rate : 0.3000
Detection Prevalence : 0.4496
Balanced Accuracy : 0.7651
'Positive' Class : 1
compute_aupr(image_level$filter_150_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')
compute_auroc(image_level$filter_150_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')
confusion_matrix(image_level, 'filter_150_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 702 167
1 125 356
Accuracy : 0.7837
95% CI : (0.7608, 0.8054)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2e-16
Kappa : 0.5375
Mcnemar's Test P-Value : 0.01642
Sensitivity : 0.6807
Specificity : 0.8489
Pos Pred Value : 0.7401
Neg Pred Value : 0.8078
Prevalence : 0.3874
Detection Rate : 0.2637
Detection Prevalence : 0.3563
Balanced Accuracy : 0.7648
'Positive' Class : 1
compute_aupr(image_level$filter_150_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')
compute_auroc(image_level$filter_150_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')
confusion_matrix(image_level, 'filter_150_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 641 162
1 186 361
Accuracy : 0.7422
95% CI : (0.718, 0.7654)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.4614
Mcnemar's Test P-Value : 0.2176
Sensitivity : 0.6902
Specificity : 0.7751
Pos Pred Value : 0.6600
Neg Pred Value : 0.7983
Prevalence : 0.3874
Detection Rate : 0.2674
Detection Prevalence : 0.4052
Balanced Accuracy : 0.7327
'Positive' Class : 1
Segmentations: zero all pixels < 200
compute_aupr(image_level$filter_200, image_level$label, 'PR: Segmentations: zero all pixels < 200')
compute_auroc(image_level$filter_200, image_level$label, 'ROC: Segmentations: zero all pixels < 200')
confusion_matrix(image_level, 'filter_200')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 548 121
1 279 402
Accuracy : 0.7037
95% CI : (0.6786, 0.728)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.64e-12
Kappa : 0.4086
Mcnemar's Test P-Value : 4.16e-15
Sensitivity : 0.7686
Specificity : 0.6626
Pos Pred Value : 0.5903
Neg Pred Value : 0.8191
Prevalence : 0.3874
Detection Rate : 0.2978
Detection Prevalence : 0.5044
Balanced Accuracy : 0.7156
'Positive' Class : 1
compute_aupr(image_level$filter_200_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')
compute_auroc(image_level$filter_200_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')
confusion_matrix(image_level, 'filter_200_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 661 233
1 166 290
Accuracy : 0.7044
95% CI : (0.6793, 0.7287)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.083e-12
Kappa : 0.3623
Mcnemar's Test P-Value : 0.0009527
Sensitivity : 0.5545
Specificity : 0.7993
Pos Pred Value : 0.6360
Neg Pred Value : 0.7394
Prevalence : 0.3874
Detection Rate : 0.2148
Detection Prevalence : 0.3378
Balanced Accuracy : 0.6769
'Positive' Class : 1
compute_aupr(image_level$filter_200_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')
compute_auroc(image_level$filter_200_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')
confusion_matrix(image_level, 'filter_200_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 766 365
1 61 158
Accuracy : 0.6844
95% CI : (0.6589, 0.7092)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 2.354e-08
Kappa : 0.2557
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.3021
Specificity : 0.9262
Pos Pred Value : 0.7215
Neg Pred Value : 0.6773
Prevalence : 0.3874
Detection Rate : 0.1170
Detection Prevalence : 0.1622
Balanced Accuracy : 0.6142
'Positive' Class : 1
Segmentations: zero all pixels < 210
compute_aupr(image_level$filter_210, image_level$label, 'PR: Segmentations: zero all pixels < 210')
compute_auroc(image_level$filter_210, image_level$label, 'ROC: Segmentations: zero all pixels < 210')
confusion_matrix(image_level, 'filter_210')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 712 326
1 115 197
Accuracy : 0.6733
95% CI : (0.6476, 0.6983)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 2.096e-06
Kappa : 0.2566
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.3767
Specificity : 0.8609
Pos Pred Value : 0.6314
Neg Pred Value : 0.6859
Prevalence : 0.3874
Detection Rate : 0.1459
Detection Prevalence : 0.2311
Balanced Accuracy : 0.6188
'Positive' Class : 1
compute_aupr(image_level$filter_210_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')
compute_auroc(image_level$filter_210_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')
confusion_matrix(image_level, 'filter_210_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 639 203
1 188 320
Accuracy : 0.7104
95% CI : (0.6854, 0.7345)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 3.463e-14
Kappa : 0.3866
Mcnemar's Test P-Value : 0.4789
Sensitivity : 0.6119
Specificity : 0.7727
Pos Pred Value : 0.6299
Neg Pred Value : 0.7589
Prevalence : 0.3874
Detection Rate : 0.2370
Detection Prevalence : 0.3763
Balanced Accuracy : 0.6923
'Positive' Class : 1
compute_aupr(image_level$filter_210_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')
compute_auroc(image_level$filter_210_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')
confusion_matrix(image_level, 'filter_210_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 562 160
1 265 363
Accuracy : 0.6852
95% CI : (0.6597, 0.7099)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.699e-08
Kappa : 0.3603
Mcnemar's Test P-Value : 4.541e-07
Sensitivity : 0.6941
Specificity : 0.6796
Pos Pred Value : 0.5780
Neg Pred Value : 0.7784
Prevalence : 0.3874
Detection Rate : 0.2689
Detection Prevalence : 0.4652
Balanced Accuracy : 0.6868
'Positive' Class : 1
Segmentations: zero all pixels < 220
compute_aupr(image_level$filter_220, image_level$label, 'PR: Segmentations: zero all pixels < 220')
compute_auroc(image_level$filter_220, image_level$label, 'ROC: Segmentations: zero all pixels < 220')
confusion_matrix(image_level, 'filter_220')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 576 224
1 251 299
Accuracy : 0.6481
95% CI : (0.622, 0.6736)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.003819
Kappa : 0.2657
Mcnemar's Test P-Value : 0.232884
Sensitivity : 0.5717
Specificity : 0.6965
Pos Pred Value : 0.5436
Neg Pred Value : 0.7200
Prevalence : 0.3874
Detection Rate : 0.2215
Detection Prevalence : 0.4074
Balanced Accuracy : 0.6341
'Positive' Class : 1
compute_aupr(image_level$filter_220_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')
compute_auroc(image_level$filter_220_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')
confusion_matrix(image_level, 'filter_220_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 545 162
1 282 361
Accuracy : 0.6711
95% CI : (0.6453, 0.6961)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 4.710e-06
Kappa : 0.3351
Mcnemar's Test P-Value : 1.628e-08
Sensitivity : 0.6902
Specificity : 0.6590
Pos Pred Value : 0.5614
Neg Pred Value : 0.7709
Prevalence : 0.3874
Detection Rate : 0.2674
Detection Prevalence : 0.4763
Balanced Accuracy : 0.6746
'Positive' Class : 1
compute_aupr(image_level$filter_220_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')
compute_auroc(image_level$filter_220_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')
confusion_matrix(image_level, 'filter_220_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 800 450
1 27 73
Accuracy : 0.6467
95% CI : (0.6205, 0.6722)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.005317
Kappa : 0.1256
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.13958
Specificity : 0.96735
Pos Pred Value : 0.73000
Neg Pred Value : 0.64000
Prevalence : 0.38741
Detection Rate : 0.05407
Detection Prevalence : 0.07407
Balanced Accuracy : 0.55347
'Positive' Class : 1
Segmentations: zero all pixels < 230
compute_aupr(image_level$filter_230, image_level$label, 'PR: Segmentations: zero all pixels < 230')
compute_auroc(image_level$filter_230, image_level$label, 'ROC: Segmentations: zero all pixels < 230')
confusion_matrix(image_level, 'filter_230')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 464 160
1 363 363
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0.2381
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.6941
Specificity : 0.5611
Pos Pred Value : 0.5000
Neg Pred Value : 0.7436
Prevalence : 0.3874
Detection Rate : 0.2689
Detection Prevalence : 0.5378
Balanced Accuracy : 0.6276
'Positive' Class : 1
compute_aupr(image_level$filter_230_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')
compute_auroc(image_level$filter_230_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')
confusion_matrix(image_level, 'filter_230_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 246 46
1 581 477
Accuracy : 0.5356
95% CI : (0.5085, 0.5624)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1764
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.9120
Specificity : 0.2975
Pos Pred Value : 0.4509
Neg Pred Value : 0.8425
Prevalence : 0.3874
Detection Rate : 0.3533
Detection Prevalence : 0.7837
Balanced Accuracy : 0.6048
'Positive' Class : 1
compute_aupr(image_level$filter_230_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')
compute_auroc(image_level$filter_230_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')
confusion_matrix(image_level, 'filter_230_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 365 110
1 462 413
Accuracy : 0.5763
95% CI : (0.5494, 0.6028)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.997
Kappa : 0.2056
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.7897
Specificity : 0.4414
Pos Pred Value : 0.4720
Neg Pred Value : 0.7684
Prevalence : 0.3874
Detection Rate : 0.3059
Detection Prevalence : 0.6481
Balanced Accuracy : 0.6155
'Positive' Class : 1
Segmentations: zero all pixels < 240
compute_aupr(image_level$filter_240, image_level$label, 'PR: Segmentations: zero all pixels < 240')
compute_auroc(image_level$filter_240, image_level$label, 'ROC: Segmentations: zero all pixels < 240')
confusion_matrix(image_level, 'filter_240')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 414 183
1 413 340
Accuracy : 0.5585
95% CI : (0.5316, 0.5852)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1394
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.6501
Specificity : 0.5006
Pos Pred Value : 0.4515
Neg Pred Value : 0.6935
Prevalence : 0.3874
Detection Rate : 0.2519
Detection Prevalence : 0.5578
Balanced Accuracy : 0.5754
'Positive' Class : 1
compute_aupr(image_level$filter_240_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')
compute_auroc(image_level$filter_240_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')
confusion_matrix(image_level, 'filter_240_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 258 83
1 569 440
Accuracy : 0.517
95% CI : (0.49, 0.544)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1309
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.8413
Specificity : 0.3120
Pos Pred Value : 0.4361
Neg Pred Value : 0.7566
Prevalence : 0.3874
Detection Rate : 0.3259
Detection Prevalence : 0.7474
Balanced Accuracy : 0.5766
'Positive' Class : 1
compute_aupr(image_level$filter_240_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')
compute_auroc(image_level$filter_240_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')
confusion_matrix(image_level, 'filter_240_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 279 103
1 548 420
Accuracy : 0.5178
95% CI : (0.4907, 0.5447)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1214
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.8031
Specificity : 0.3374
Pos Pred Value : 0.4339
Neg Pred Value : 0.7304
Prevalence : 0.3874
Detection Rate : 0.3111
Detection Prevalence : 0.7170
Balanced Accuracy : 0.5702
'Positive' Class : 1
Segmentations: zero all pixels < 250
compute_aupr(image_level$filter_250, image_level$label, 'PR: Segmentations: zero all pixels < 250')
compute_auroc(image_level$filter_250, image_level$label, 'ROC: Segmentations: zero all pixels < 250')
confusion_matrix(image_level, 'filter_250')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 51 9
1 776 514
Accuracy : 0.4185
95% CI : (0.392, 0.4454)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.035
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.98279
Specificity : 0.06167
Pos Pred Value : 0.39845
Neg Pred Value : 0.85000
Prevalence : 0.38741
Detection Rate : 0.38074
Detection Prevalence : 0.95556
Balanced Accuracy : 0.52223
'Positive' Class : 1
compute_aupr(image_level$filter_250_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')
compute_auroc(image_level$filter_250_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')
confusion_matrix(image_level, 'filter_250_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 50 8
1 777 515
Accuracy : 0.4185
95% CI : (0.392, 0.4454)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.0356
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.98470
Specificity : 0.06046
Pos Pred Value : 0.39861
Neg Pred Value : 0.86207
Prevalence : 0.38741
Detection Rate : 0.38148
Detection Prevalence : 0.95704
Balanced Accuracy : 0.52258
'Positive' Class : 1
compute_aupr(image_level$filter_250_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')
compute_auroc(image_level$filter_250_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')
confusion_matrix(image_level, 'filter_250_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 53 9
1 774 514
Accuracy : 0.42
95% CI : (0.3935, 0.4468)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.0369
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.98279
Specificity : 0.06409
Pos Pred Value : 0.39907
Neg Pred Value : 0.85484
Prevalence : 0.38741
Detection Rate : 0.38074
Detection Prevalence : 0.95407
Balanced Accuracy : 0.52344
'Positive' Class : 1
Segmentations: zero all pixels < 257
compute_aupr(image_level$filter_257, image_level$label, 'PR: Segmentations: zero all pixels < 257')
compute_auroc(image_level$filter_257, image_level$label, 'ROC: Segmentations: zero all pixels < 257')
confusion_matrix(image_level, 'filter_257')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 827 523
1 0 0
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6126
Prevalence : 0.3874
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(image_level$filter_257_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')
compute_auroc(image_level$filter_257_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')
confusion_matrix(image_level, 'filter_257_binarized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 827 523
1 0 0
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6126
Prevalence : 0.3874
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(image_level$filter_257_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')
compute_auroc(image_level$filter_257_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')
confusion_matrix(image_level, 'filter_257_skeletonized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 827 523
1 0 0
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6126
Prevalence : 0.3874
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels > 10
compute_aupr(image_level$filter_10, image_level$label, 'PR: Segmentations: zero all pixels < 10')
compute_auroc(image_level$filter_10, image_level$label, 'ROC: Segmentations: zero all pixels < 10')
confusion_matrix(image_level, 'filter_10')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 763 196
1 64 327
Accuracy : 0.8074
95% CI : (0.7853, 0.8281)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2e-16
Kappa : 0.5745
Mcnemar's Test P-Value : 4.5e-16
Sensitivity : 0.6252
Specificity : 0.9226
Pos Pred Value : 0.8363
Neg Pred Value : 0.7956
Prevalence : 0.3874
Detection Rate : 0.2422
Detection Prevalence : 0.2896
Balanced Accuracy : 0.7739
'Positive' Class : 1
compute_aupr(image_level$filter_10_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')
compute_auroc(image_level$filter_10_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')
confusion_matrix(image_level, 'filter_10_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 621 7
1 206 516
Accuracy : 0.8422
95% CI : (0.8217, 0.8613)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.6893
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.9866
Specificity : 0.7509
Pos Pred Value : 0.7147
Neg Pred Value : 0.9889
Prevalence : 0.3874
Detection Rate : 0.3822
Detection Prevalence : 0.5348
Balanced Accuracy : 0.8688
'Positive' Class : 1
compute_aupr(image_level$filter_10_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')
compute_auroc(image_level$filter_10_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')
confusion_matrix(image_level, 'filter_10_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 696 43
1 131 480
Accuracy : 0.8711
95% CI : (0.8521, 0.8885)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.7366
Mcnemar's Test P-Value : 4.24e-11
Sensitivity : 0.9178
Specificity : 0.8416
Pos Pred Value : 0.7856
Neg Pred Value : 0.9418
Prevalence : 0.3874
Detection Rate : 0.3556
Detection Prevalence : 0.4526
Balanced Accuracy : 0.8797
'Positive' Class : 1
Segmentations: zero all pixels < 75 and > 150
compute_aupr(image_level$filter_75, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')
compute_auroc(image_level$filter_75, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')
confusion_matrix(image_level, 'filter_75')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 597 77
1 230 446
Accuracy : 0.7726
95% CI : (0.7493, 0.7947)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5453
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.8528
Specificity : 0.7219
Pos Pred Value : 0.6598
Neg Pred Value : 0.8858
Prevalence : 0.3874
Detection Rate : 0.3304
Detection Prevalence : 0.5007
Balanced Accuracy : 0.7873
'Positive' Class : 1
compute_aupr(image_level$filter_75_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')
compute_auroc(image_level$filter_75_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')
confusion_matrix(image_level, 'filter_75_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 545 69
1 282 454
Accuracy : 0.74
95% CI : (0.7157, 0.7632)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.4904
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.8681
Specificity : 0.6590
Pos Pred Value : 0.6168
Neg Pred Value : 0.8876
Prevalence : 0.3874
Detection Rate : 0.3363
Detection Prevalence : 0.5452
Balanced Accuracy : 0.7635
'Positive' Class : 1
compute_aupr(image_level$filter_75_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
compute_auroc(image_level$filter_75_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
confusion_matrix(image_level, 'filter_75_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 726 224
1 101 299
Accuracy : 0.7593
95% CI : (0.7355, 0.7819)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.4699
Mcnemar's Test P-Value : 1.312e-11
Sensitivity : 0.5717
Specificity : 0.8779
Pos Pred Value : 0.7475
Neg Pred Value : 0.7642
Prevalence : 0.3874
Detection Rate : 0.2215
Detection Prevalence : 0.2963
Balanced Accuracy : 0.7248
'Positive' Class : 1
Subject-level Analysis
RetCam Images
compute_aupr(subject_level$retcam, subject_level$label, 'PR: Raw RetCam Images')
compute_auroc(subject_level$retcam, subject_level$label, 'ROC: Raw RetCam Images')
confusion_matrix(subject_level, 'retcam')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 0
1 0 27
Accuracy : 1
95% CI : (0.9487, 1)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.534e-15
Kappa : 1
Mcnemar's Test P-Value : NA
Sensitivity : 1.0000
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.3857
Balanced Accuracy : 1.0000
'Positive' Class : 1
compute_aupr(subject_level$retcam_random, subject_level$label, 'PR: Raw RetCam Images - Shuffled Labels')
compute_auroc(subject_level$retcam_random, subject_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')
confusion_matrix(subject_level, 'retcam_random')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 24 2
1 19 25
Accuracy : 0.7
95% CI : (0.5787, 0.8038)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0869051
Kappa : 0.4333
Mcnemar's Test P-Value : 0.0004803
Sensitivity : 0.9259
Specificity : 0.5581
Pos Pred Value : 0.5682
Neg Pred Value : 0.9231
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.6286
Balanced Accuracy : 0.7420
'Positive' Class : 1
Segmentations: zero all pixels < 0
compute_aupr(subject_level$filter_0, subject_level$label, 'PR: Segmentations: zero all pixels < 0')
compute_auroc(subject_level$filter_0, subject_level$label, 'ROC: Segmentations: zero all pixels < 0')
confusion_matrix(subject_level, 'filter_0')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 42 2
1 1 25
Accuracy : 0.9571
95% CI : (0.8798, 0.9911)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.232e-11
Kappa : 0.9089
Mcnemar's Test P-Value : 1
Sensitivity : 0.9259
Specificity : 0.9767
Pos Pred Value : 0.9615
Neg Pred Value : 0.9545
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.3714
Balanced Accuracy : 0.9513
'Positive' Class : 1
compute_aupr(subject_level$filter_0_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')
compute_auroc(subject_level$filter_0_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')
confusion_matrix(subject_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 41 0
1 2 27
Accuracy : 0.9714
95% CI : (0.9006, 0.9965)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.53e-12
Kappa : 0.9405
Mcnemar's Test P-Value : 0.4795
Sensitivity : 1.0000
Specificity : 0.9535
Pos Pred Value : 0.9310
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4143
Balanced Accuracy : 0.9767
'Positive' Class : 1
compute_aupr(subject_level$filter_0_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')
compute_auroc(subject_level$filter_0_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')
confusion_matrix(subject_level, 'filter_0_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 40 0
1 3 27
Accuracy : 0.9571
95% CI : (0.8798, 0.9911)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.232e-11
Kappa : 0.9114
Mcnemar's Test P-Value : 0.2482
Sensitivity : 1.0000
Specificity : 0.9302
Pos Pred Value : 0.9000
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4286
Balanced Accuracy : 0.9651
'Positive' Class : 1
compute_aupr(subject_level$filter_0_random, subject_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')
compute_auroc(subject_level$filter_0_random, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')
confusion_matrix(subject_level, 'filter_0_random')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels < 50
compute_aupr(subject_level$filter_50, subject_level$label, 'PR: Segmentations: zero all pixels < 50')
compute_auroc(subject_level$filter_50, subject_level$label, 'ROC: Segmentations: zero all pixels < 50')
confusion_matrix(subject_level, 'filter_50')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 36 0
1 7 27
Accuracy : 0.9
95% CI : (0.8048, 0.9588)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 8.517e-08
Kappa : 0.7987
Mcnemar's Test P-Value : 0.02334
Sensitivity : 1.0000
Specificity : 0.8372
Pos Pred Value : 0.7941
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4857
Balanced Accuracy : 0.9186
'Positive' Class : 1
compute_aupr(subject_level$filter_50_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')
compute_auroc(subject_level$filter_50_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')
confusion_matrix(subject_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 41 0
1 2 27
Accuracy : 0.9714
95% CI : (0.9006, 0.9965)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.53e-12
Kappa : 0.9405
Mcnemar's Test P-Value : 0.4795
Sensitivity : 1.0000
Specificity : 0.9535
Pos Pred Value : 0.9310
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4143
Balanced Accuracy : 0.9767
'Positive' Class : 1
compute_aupr(subject_level$filter_50_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')
compute_auroc(subject_level$filter_50_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')
confusion_matrix(subject_level, 'filter_50_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 19 0
1 24 27
Accuracy : 0.6571
95% CI : (0.534, 0.7665)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.2718
Kappa : 0.3792
Mcnemar's Test P-Value : 2.668e-06
Sensitivity : 1.0000
Specificity : 0.4419
Pos Pred Value : 0.5294
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.7286
Balanced Accuracy : 0.7209
'Positive' Class : 1
Segmentations: zero all pixels < 100
compute_aupr(subject_level$filter_100, subject_level$label, 'PR: Segmentations: zero all pixels < 100')
compute_auroc(subject_level$filter_100, subject_level$label, 'ROC: Segmentations: zero all pixels < 100')
confusion_matrix(subject_level, 'filter_100')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 10
1 0 17
Accuracy : 0.8571
95% CI : (0.7529, 0.9293)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 7.748e-06
Kappa : 0.6762
Mcnemar's Test P-Value : 0.004427
Sensitivity : 0.6296
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.8113
Prevalence : 0.3857
Detection Rate : 0.2429
Detection Prevalence : 0.2429
Balanced Accuracy : 0.8148
'Positive' Class : 1
compute_aupr(subject_level$filter_100_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')
compute_auroc(subject_level$filter_100_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')
confusion_matrix(subject_level, 'filter_100_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 39 4
1 4 23
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7588
Mcnemar's Test P-Value : 1
Sensitivity : 0.8519
Specificity : 0.9070
Pos Pred Value : 0.8519
Neg Pred Value : 0.9070
Prevalence : 0.3857
Detection Rate : 0.3286
Detection Prevalence : 0.3857
Balanced Accuracy : 0.8794
'Positive' Class : 1
compute_aupr(subject_level$filter_100_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')
compute_auroc(subject_level$filter_100_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')
confusion_matrix(subject_level, 'filter_100_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 38 3
1 5 24
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7621
Mcnemar's Test P-Value : 0.7237
Sensitivity : 0.8889
Specificity : 0.8837
Pos Pred Value : 0.8276
Neg Pred Value : 0.9268
Prevalence : 0.3857
Detection Rate : 0.3429
Detection Prevalence : 0.4143
Balanced Accuracy : 0.8863
'Positive' Class : 1
Segmentations: zero all pixels < 150
compute_aupr(subject_level$filter_150, subject_level$label, 'PR: Segmentations: zero all pixels < 150')
compute_auroc(subject_level$filter_150, subject_level$label, 'ROC: Segmentations: zero all pixels < 150')
confusion_matrix(subject_level, 'filter_150')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 37 0
1 6 27
Accuracy : 0.9143
95% CI : (0.8227, 0.9679)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.438e-08
Kappa : 0.8263
Mcnemar's Test P-Value : 0.04123
Sensitivity : 1.0000
Specificity : 0.8605
Pos Pred Value : 0.8182
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4714
Balanced Accuracy : 0.9302
'Positive' Class : 1
compute_aupr(subject_level$filter_150_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')
compute_auroc(subject_level$filter_150_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')
confusion_matrix(subject_level, 'filter_150_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 5
1 0 22
Accuracy : 0.9286
95% CI : (0.8411, 0.9764)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.054e-09
Kappa : 0.8439
Mcnemar's Test P-Value : 0.07364
Sensitivity : 0.8148
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.8958
Prevalence : 0.3857
Detection Rate : 0.3143
Detection Prevalence : 0.3143
Balanced Accuracy : 0.9074
'Positive' Class : 1
compute_aupr(subject_level$filter_150_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')
compute_auroc(subject_level$filter_150_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')
confusion_matrix(subject_level, 'filter_150_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 38 3
1 5 24
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7621
Mcnemar's Test P-Value : 0.7237
Sensitivity : 0.8889
Specificity : 0.8837
Pos Pred Value : 0.8276
Neg Pred Value : 0.9268
Prevalence : 0.3857
Detection Rate : 0.3429
Detection Prevalence : 0.4143
Balanced Accuracy : 0.8863
'Positive' Class : 1
Segmentations: zero all pixels < 200
compute_aupr(subject_level$filter_200, subject_level$label, 'PR: Segmentations: zero all pixels < 200')
compute_auroc(subject_level$filter_200, subject_level$label, 'ROC: Segmentations: zero all pixels < 200')
confusion_matrix(subject_level, 'filter_200')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 34 4
1 9 23
Accuracy : 0.8143
95% CI : (0.7034, 0.8972)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0002607
Kappa : 0.6211
Mcnemar's Test P-Value : 0.2672575
Sensitivity : 0.8519
Specificity : 0.7907
Pos Pred Value : 0.7187
Neg Pred Value : 0.8947
Prevalence : 0.3857
Detection Rate : 0.3286
Detection Prevalence : 0.4571
Balanced Accuracy : 0.8213
'Positive' Class : 1
compute_aupr(subject_level$filter_200_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')
compute_auroc(subject_level$filter_200_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')
confusion_matrix(subject_level, 'filter_200_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 40 12
1 3 15
Accuracy : 0.7857
95% CI : (0.6713, 0.8748)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.001729
Kappa : 0.5179
Mcnemar's Test P-Value : 0.038867
Sensitivity : 0.5556
Specificity : 0.9302
Pos Pred Value : 0.8333
Neg Pred Value : 0.7692
Prevalence : 0.3857
Detection Rate : 0.2143
Detection Prevalence : 0.2571
Balanced Accuracy : 0.7429
'Positive' Class : 1
compute_aupr(subject_level$filter_200_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')
compute_auroc(subject_level$filter_200_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')
confusion_matrix(subject_level, 'filter_200_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 25
1 0 2
Accuracy : 0.6429
95% CI : (0.5193, 0.7539)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.3595
Kappa : 0.0895
Mcnemar's Test P-Value : 1.587e-06
Sensitivity : 0.07407
Specificity : 1.00000
Pos Pred Value : 1.00000
Neg Pred Value : 0.63235
Prevalence : 0.38571
Detection Rate : 0.02857
Detection Prevalence : 0.02857
Balanced Accuracy : 0.53704
'Positive' Class : 1
Segmentations: zero all pixels < 210
compute_aupr(subject_level$filter_210, subject_level$label, 'PR: Segmentations: zero all pixels < 210')
compute_auroc(subject_level$filter_210, subject_level$label, 'ROC: Segmentations: zero all pixels < 210')
confusion_matrix(subject_level, 'filter_210')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 20
1 0 7
Accuracy : 0.7143
95% CI : (0.5938, 0.816)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0532
Kappa : 0.3007
Mcnemar's Test P-Value : 2.152e-05
Sensitivity : 0.2593
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.6825
Prevalence : 0.3857
Detection Rate : 0.1000
Detection Prevalence : 0.1000
Balanced Accuracy : 0.6296
'Positive' Class : 1
compute_aupr(subject_level$filter_210_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')
compute_auroc(subject_level$filter_210_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')
confusion_matrix(subject_level, 'filter_210_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 39 9
1 4 18
Accuracy : 0.8143
95% CI : (0.7034, 0.8972)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0002607
Kappa : 0.5941
Mcnemar's Test P-Value : 0.2672575
Sensitivity : 0.6667
Specificity : 0.9070
Pos Pred Value : 0.8182
Neg Pred Value : 0.8125
Prevalence : 0.3857
Detection Rate : 0.2571
Detection Prevalence : 0.3143
Balanced Accuracy : 0.7868
'Positive' Class : 1
compute_aupr(subject_level$filter_210_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')
compute_auroc(subject_level$filter_210_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')
confusion_matrix(subject_level, 'filter_210_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 35 7
1 8 20
Accuracy : 0.7857
95% CI : (0.6713, 0.8748)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.001729
Kappa : 0.5509
Mcnemar's Test P-Value : 1.000000
Sensitivity : 0.7407
Specificity : 0.8140
Pos Pred Value : 0.7143
Neg Pred Value : 0.8333
Prevalence : 0.3857
Detection Rate : 0.2857
Detection Prevalence : 0.4000
Balanced Accuracy : 0.7773
'Positive' Class : 1
Segmentations: zero all pixels < 220
compute_aupr(subject_level$filter_220, subject_level$label, 'PR: Segmentations: zero all pixels < 220')
compute_auroc(subject_level$filter_220, subject_level$label, 'ROC: Segmentations: zero all pixels < 220')
confusion_matrix(subject_level, 'filter_220')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 34 12
1 9 15
Accuracy : 0.7
95% CI : (0.5787, 0.8038)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.08691
Kappa : 0.3536
Mcnemar's Test P-Value : 0.66252
Sensitivity : 0.5556
Specificity : 0.7907
Pos Pred Value : 0.6250
Neg Pred Value : 0.7391
Prevalence : 0.3857
Detection Rate : 0.2143
Detection Prevalence : 0.3429
Balanced Accuracy : 0.6731
'Positive' Class : 1
compute_aupr(subject_level$filter_220_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')
compute_auroc(subject_level$filter_220_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')
confusion_matrix(subject_level, 'filter_220_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 31 7
1 12 20
Accuracy : 0.7286
95% CI : (0.609, 0.828)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.03066
Kappa : 0.4463
Mcnemar's Test P-Value : 0.35880
Sensitivity : 0.7407
Specificity : 0.7209
Pos Pred Value : 0.6250
Neg Pred Value : 0.8158
Prevalence : 0.3857
Detection Rate : 0.2857
Detection Prevalence : 0.4571
Balanced Accuracy : 0.7308
'Positive' Class : 1
compute_aupr(subject_level$filter_220_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')
compute_auroc(subject_level$filter_220_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')
confusion_matrix(subject_level, 'filter_220_skeletonized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels < 230
compute_aupr(subject_level$filter_230, subject_level$label, 'PR: Segmentations: zero all pixels < 230')
compute_auroc(subject_level$filter_230, subject_level$label, 'ROC: Segmentations: zero all pixels < 230')
confusion_matrix(subject_level, 'filter_230')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 26 6
1 17 21
Accuracy : 0.6714
95% CI : (0.5488, 0.7791)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.19581
Kappa : 0.3555
Mcnemar's Test P-Value : 0.03706
Sensitivity : 0.7778
Specificity : 0.6047
Pos Pred Value : 0.5526
Neg Pred Value : 0.8125
Prevalence : 0.3857
Detection Rate : 0.3000
Detection Prevalence : 0.5429
Balanced Accuracy : 0.6912
'Positive' Class : 1
compute_aupr(subject_level$filter_230_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')
compute_auroc(subject_level$filter_230_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')
confusion_matrix(subject_level, 'filter_230_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 10 1
1 33 26
Accuracy : 0.5143
95% CI : (0.3917, 0.6356)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.966
Kappa : 0.1602
Mcnemar's Test P-Value : 1.058e-07
Sensitivity : 0.9630
Specificity : 0.2326
Pos Pred Value : 0.4407
Neg Pred Value : 0.9091
Prevalence : 0.3857
Detection Rate : 0.3714
Detection Prevalence : 0.8429
Balanced Accuracy : 0.5978
'Positive' Class : 1
compute_aupr(subject_level$filter_230_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')
compute_auroc(subject_level$filter_230_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')
confusion_matrix(subject_level, 'filter_230_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 17 2
1 26 25
Accuracy : 0.6
95% CI : (0.4759, 0.7153)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.6466
Kappa : 0.2757
Mcnemar's Test P-Value : 1.383e-05
Sensitivity : 0.9259
Specificity : 0.3953
Pos Pred Value : 0.4902
Neg Pred Value : 0.8947
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.7286
Balanced Accuracy : 0.6606
'Positive' Class : 1
Segmentations: zero all pixels < 240
compute_aupr(subject_level$filter_240, subject_level$label, 'PR: Segmentations: zero all pixels < 240')
compute_auroc(subject_level$filter_240, subject_level$label, 'ROC: Segmentations: zero all pixels < 240')
confusion_matrix(subject_level, 'filter_240')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 23 9
1 20 18
Accuracy : 0.5857
95% CI : (0.4617, 0.7023)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.73214
Kappa : 0.1873
Mcnemar's Test P-Value : 0.06332
Sensitivity : 0.6667
Specificity : 0.5349
Pos Pred Value : 0.4737
Neg Pred Value : 0.7187
Prevalence : 0.3857
Detection Rate : 0.2571
Detection Prevalence : 0.5429
Balanced Accuracy : 0.6008
'Positive' Class : 1
compute_aupr(subject_level$filter_240_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')
compute_auroc(subject_level$filter_240_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')
confusion_matrix(subject_level, 'filter_240_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 12 1
1 31 26
Accuracy : 0.5429
95% CI : (0.4194, 0.6626)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9106
Kappa : 0.2006
Mcnemar's Test P-Value : 2.951e-07
Sensitivity : 0.9630
Specificity : 0.2791
Pos Pred Value : 0.4561
Neg Pred Value : 0.9231
Prevalence : 0.3857
Detection Rate : 0.3714
Detection Prevalence : 0.8143
Balanced Accuracy : 0.6210
'Positive' Class : 1
compute_aupr(subject_level$filter_240_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')
compute_auroc(subject_level$filter_240_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')
confusion_matrix(subject_level, 'filter_240_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 14 2
1 29 25
Accuracy : 0.5571
95% CI : (0.4334, 0.6759)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.8651
Kappa : 0.2121
Mcnemar's Test P-Value : 3.016e-06
Sensitivity : 0.9259
Specificity : 0.3256
Pos Pred Value : 0.4630
Neg Pred Value : 0.8750
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.7714
Balanced Accuracy : 0.6258
'Positive' Class : 1
Segmentations: zero all pixels < 250
compute_aupr(subject_level$filter_250, subject_level$label, 'PR: Segmentations: zero all pixels < 250')
compute_auroc(subject_level$filter_250, subject_level$label, 'ROC: Segmentations: zero all pixels < 250')
confusion_matrix(subject_level, 'filter_250')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1 0
1 42 27
Accuracy : 0.4
95% CI : (0.2847, 0.5241)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9999
Kappa : 0.018
Mcnemar's Test P-Value : 2.509e-10
Sensitivity : 1.00000
Specificity : 0.02326
Pos Pred Value : 0.39130
Neg Pred Value : 1.00000
Prevalence : 0.38571
Detection Rate : 0.38571
Detection Prevalence : 0.98571
Balanced Accuracy : 0.51163
'Positive' Class : 1
compute_aupr(subject_level$filter_250_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')
compute_auroc(subject_level$filter_250_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')
confusion_matrix(subject_level, 'filter_250_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1 0
1 42 27
Accuracy : 0.4
95% CI : (0.2847, 0.5241)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9999
Kappa : 0.018
Mcnemar's Test P-Value : 2.509e-10
Sensitivity : 1.00000
Specificity : 0.02326
Pos Pred Value : 0.39130
Neg Pred Value : 1.00000
Prevalence : 0.38571
Detection Rate : 0.38571
Detection Prevalence : 0.98571
Balanced Accuracy : 0.51163
'Positive' Class : 1
compute_aupr(subject_level$filter_250_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')
compute_auroc(subject_level$filter_250_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')
confusion_matrix(subject_level, 'filter_250_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1 0
1 42 27
Accuracy : 0.4
95% CI : (0.2847, 0.5241)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9999
Kappa : 0.018
Mcnemar's Test P-Value : 2.509e-10
Sensitivity : 1.00000
Specificity : 0.02326
Pos Pred Value : 0.39130
Neg Pred Value : 1.00000
Prevalence : 0.38571
Detection Rate : 0.38571
Detection Prevalence : 0.98571
Balanced Accuracy : 0.51163
'Positive' Class : 1
Segmentations: zero all pixels < 257
compute_aupr(subject_level$filter_257, subject_level$label, 'PR: Segmentations: zero all pixels < 257')
compute_auroc(subject_level$filter_257, subject_level$label, 'ROC: Segmentations: zero all pixels < 257')
confusion_matrix(subject_level, 'filter_257')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(subject_level$filter_257_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')
compute_auroc(subject_level$filter_257_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')
confusion_matrix(subject_level, 'filter_257_binarized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(subject_level$filter_257_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')
compute_auroc(subject_level$filter_257_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')
confusion_matrix(subject_level, 'filter_257_skeletonized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels > 10
compute_aupr(subject_level$filter_10, subject_level$label, 'PR: Segmentations: zero all pixels < 10')
compute_auroc(subject_level$filter_10, subject_level$label, 'ROC: Segmentations: zero all pixels < 10')
confusion_matrix(subject_level, 'filter_10')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 41 8
1 2 19
Accuracy : 0.8571
95% CI : (0.7529, 0.9293)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 7.748e-06
Kappa : 0.6855
Mcnemar's Test P-Value : 0.1138
Sensitivity : 0.7037
Specificity : 0.9535
Pos Pred Value : 0.9048
Neg Pred Value : 0.8367
Prevalence : 0.3857
Detection Rate : 0.2714
Detection Prevalence : 0.3000
Balanced Accuracy : 0.8286
'Positive' Class : 1
compute_aupr(subject_level$filter_10_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')
compute_auroc(subject_level$filter_10_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')
confusion_matrix(subject_level, 'filter_10_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 34 0
1 9 27
Accuracy : 0.8714
95% CI : (0.7699, 0.9395)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.949e-06
Kappa : 0.7445
Mcnemar's Test P-Value : 0.007661
Sensitivity : 1.0000
Specificity : 0.7907
Pos Pred Value : 0.7500
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.5143
Balanced Accuracy : 0.8953
'Positive' Class : 1
compute_aupr(subject_level$filter_10_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')
compute_auroc(subject_level$filter_10_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')
confusion_matrix(subject_level, 'filter_10_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 39 0
1 4 27
Accuracy : 0.9429
95% CI : (0.8601, 0.9842)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.41e-10
Kappa : 0.8826
Mcnemar's Test P-Value : 0.1336
Sensitivity : 1.0000
Specificity : 0.9070
Pos Pred Value : 0.8710
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4429
Balanced Accuracy : 0.9535
'Positive' Class : 1
Segmentations: zero all pixels < 75 and > 150
compute_aupr(subject_level$filter_75, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')
compute_auroc(subject_level$filter_75, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')
confusion_matrix(subject_level, 'filter_75')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 36 1
1 7 26
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7684
Mcnemar's Test P-Value : 0.0771
Sensitivity : 0.9630
Specificity : 0.8372
Pos Pred Value : 0.7879
Neg Pred Value : 0.9730
Prevalence : 0.3857
Detection Rate : 0.3714
Detection Prevalence : 0.4714
Balanced Accuracy : 0.9001
'Positive' Class : 1
compute_aupr(subject_level$filter_75_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')
compute_auroc(subject_level$filter_75_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')
confusion_matrix(subject_level, 'filter_75_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 33 0
1 10 27
Accuracy : 0.8571
95% CI : (0.7529, 0.9293)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 7.748e-06
Kappa : 0.718
Mcnemar's Test P-Value : 0.004427
Sensitivity : 1.0000
Specificity : 0.7674
Pos Pred Value : 0.7297
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.5286
Balanced Accuracy : 0.8837
'Positive' Class : 1
compute_aupr(subject_level$filter_75_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
compute_auroc(subject_level$filter_75_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
confusion_matrix(subject_level, 'filter_75_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 13
1 0 14
Accuracy : 0.8143
95% CI : (0.7034, 0.8972)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0002607
Kappa : 0.5695
Mcnemar's Test P-Value : 0.0008741
Sensitivity : 0.5185
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.7679
Prevalence : 0.3857
Detection Rate : 0.2000
Detection Prevalence : 0.2000
Balanced Accuracy : 0.7593
'Positive' Class : 1
aupr_figures <- function(predictions, labels, title, names, save_name) {
n_black <- length(labels[labels == 1])
n_total <- length(labels)
for (i in seq_along(predictions)) {
preds <- prediction(predictions[i], labels)
aupr <- performance(preds, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
perf <- performance(preds, 'prec', 'rec')
if (is.null(dev.list())) {
png(save_name,
width=7,
height=5,
units='in',
res=300)
plot(perf,
main = title,
xlim=c(0,1),
ylim=c(0,1),
col = i,
lwd = 2)
} else {
plot(perf,
main = title,
xlim=c(0,1),
ylim=c(0,1),
col = i,
lwd = 2,
add = TRUE)
}
}
abline(a = n_black/n_total,
b = 0,
col = 'red',
lty = 2,
lwd = 2)
legend('bottomleft',
names,
lwd = 2,
col = seq_along(predictions),
bty = 'n',
inset = c(0.1, 0.07))
legend('bottomleft',
paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))),
lwd = 2,
lty = 2,
col = 'red',
bty = 'n',
inset = c(0.1, 0.0))
}Figures for Paper
aupr_figures(list(image_level$filter_0, image_level$filter_50, image_level$filter_200, image_level$filter_240),
image_level$label,
'Precision-Recall Curves of Thresholded RVMs',
c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
'./out/figures/thresholded_image.png')aupr_figures(list(image_level$filter_0_binarized, image_level$filter_50_binarized, image_level$filter_200_binarized, image_level$filter_240_binarized),
image_level$label,
'Precision-Recall Curves of Binarized RVMs',
c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
'./out/figures/binarized_image.png')aupr_figures(list(image_level$filter_0_skeletonized, image_level$filter_50_skeletonized, image_level$filter_200_skeletonized, image_level$filter_240_skeletonized),
image_level$label,
'Precision-Recall Curves of Skeletonized RVMs',
c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
'./out/figures/skeletonized_image.png')aupr_figures(list(subject_level$filter_0, subject_level$filter_50, subject_level$filter_200, subject_level$filter_240),
subject_level$label,
'Precision-Recall Curves of Thresholded RVMs',
c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
'./out/figures/thresholded_subject.png')aupr_figures(list(subject_level$filter_0_binarized, subject_level$filter_50_binarized, subject_level$filter_200_binarized, subject_level$filter_240_binarized),
subject_level$label,
'Precision-Recall Curves of Binarized RVMs',
c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
'./out/figures/binarized_subject.png')aupr_figures(list(subject_level$filter_0_skeletonized, subject_level$filter_50_skeletonized, subject_level$filter_200_skeletonized, subject_level$filter_240_skeletonized),
subject_level$label,
'Precision-Recall Curves of Skeletonized RVMs',
c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
'./out/figures/skeletonized_subject.png')Evaluate on Manual Segmentations
manual_binary_0 <- read_csv('./out/probabilities/manual_segmentations_binarized_0.csv') %>%
mutate(img_name = basename(img_loc)) %>%
separate(img_name, c('site', 'subject_id')) %>%
filter(!is.na(as.numeric(subject_id))) %>%
mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
select(subject_id, probability) %>%
inner_join(irop_data) %>%
filter(race == 'African American' | race == 'Caucasian/White') %>%
mutate(race = if_else(race == 'African American', 1, 0),
across(race, as.factor))
── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
img_loc = col_character(),
probability = col_double()
)Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
NAs introduced by coercionJoining, by = "subject_id"manual_skeleton_0 <- read_csv('./out/probabilities/manual_segmentations_skeletonized_0.csv') %>%
mutate(img_name = basename(img_loc)) %>%
separate(img_name, c('site', 'subject_id')) %>%
filter(!is.na(as.numeric(subject_id))) %>%
mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
select(subject_id, probability) %>%
inner_join(irop_data) %>%
filter(race == 'African American' | race == 'Caucasian/White') %>%
mutate(race = if_else(race == 'African American', 1, 0),
across(race, as.factor))
── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
img_loc = col_character(),
probability = col_double()
)Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
NAs introduced by coercionJoining, by = "subject_id"manual_binary_50 <- read_csv('./out/probabilities/manual_segmentations_binarized_50.csv') %>%
mutate(img_name = basename(img_loc)) %>%
separate(img_name, c('site', 'subject_id')) %>%
filter(!is.na(as.numeric(subject_id))) %>%
mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
select(subject_id, probability) %>%
inner_join(irop_data) %>%
filter(race == 'African American' | race == 'Caucasian/White') %>%
mutate(race = if_else(race == 'African American', 1, 0),
across(race, as.factor))
── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
img_loc = col_character(),
probability = col_double()
)Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
NAs introduced by coercionJoining, by = "subject_id"manual_skeleton_50 <- read_csv('./out/probabilities/manual_segmentations_skeletonized_50.csv') %>%
mutate(img_name = basename(img_loc)) %>%
separate(img_name, c('site', 'subject_id')) %>%
filter(!is.na(as.numeric(subject_id))) %>%
mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
select(subject_id, probability) %>%
inner_join(irop_data) %>%
filter(race == 'African American' | race == 'Caucasian/White') %>%
mutate(race = if_else(race == 'African American', 1, 0),
across(race, as.factor))
── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
img_loc = col_character(),
probability = col_double()
)Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
NAs introduced by coercionJoining, by = "subject_id"pred <- prediction(manual_binary_0$probability, manual_binary_0$race)
num_yes <- length(manual_binary_0$race[manual_binary_0$race == 1])
num_no <- length(manual_binary_0$race[manual_binary_0$race == 0])
perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Binarized - 0 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))
pred <- prediction(manual_skeleton_0$probability, manual_skeleton_0$race)
num_yes <- length(manual_skeleton_0$race[manual_skeleton_0$race == 1])
num_no <- length(manual_skeleton_0$race[manual_skeleton_0$race == 0])
perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Skeletonized - 0 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))
pred <- prediction(manual_binary_50$probability, manual_binary_50$race)
num_yes <- length(manual_binary_50$race[manual_binary_50$race == 1])
num_no <- length(manual_binary_50$race[manual_binary_50$race == 0])
perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Binarized - 50 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))
pred <- prediction(manual_skeleton_50$probability, manual_skeleton_50$race)
num_yes <- length(manual_skeleton_50$race[manual_skeleton_50$race == 1])
num_no <- length(manual_skeleton_50$race[manual_skeleton_50$race == 0])
perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Skeletonized - 50 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))
image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
select(img_loc, image_id, label, retcam = probability) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_binarized_test.csv', col_types = cols()), filter_0_binarized = probability))pred <- prediction(image_level$filter_0_binarized, image_level$label)
num_yes <- length(image_level$label[image_level$label == 1])
num_no <- length(image_level$label[image_level$label == 0])
perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Skeletonized - 0 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))
LS0tCnRpdGxlOiAnRXZhbHVhdGUgTW9kZWxzJwphdXRob3I6ICdBdXRob3I6IEFhcm9uIFMgQ295bmVyLCBQaEQnCmRhdGU6ICdMYXN0IHVwZGF0ZTogYHIgU3lzLkRhdGUoKWAnCm91dHB1dDoKICAgIGh0bWxfbm90ZWJvb2s6CiAgICAgICAgdG9jOiB5ZXMKICAgICAgICB0b2NfZmxvYXQ6IHllcwogICAgICAgIHRvY19kZXB0aDogMwotLS0KCgojIFNldHVwCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShqYW5pdG9yKQpsaWJyYXJ5KHRvb2xzKQpsaWJyYXJ5KGNhcmV0KQpsaWJyYXJ5KFJPQ1IpCmxpYnJhcnkoY2FyZXQpCmBgYAoKCmBgYHtyfQpjb21wdXRlX2F1cm9jIDwtIGZ1bmN0aW9uKHByZWRpY3Rpb25zLCBsYWJlbHMsIHRpdGxlKSB7CiAgICBwcmVkIDwtIHByZWRpY3Rpb24ocHJlZGljdGlvbnMsIGxhYmVscykKICAgIGF1cm9jIDwtIHBlcmZvcm1hbmNlKHByZWQsIG1lYXN1cmUgPSAnYXVjJykKICAgIGF1cm9jIDwtIGF1cm9jQHkudmFsdWVzW1sxXV0KICAgIHBlcmYgPC0gcGVyZm9ybWFuY2UocHJlZCwgJ3RwcicsICdmcHInKQogICAgcGxvdChwZXJmLAogICAgICAgICBtYWluID0gdGl0bGUsCiAgICAgICAgIHhsaW09YygwLDEpLAogICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgY29sb3JpemUgPSBGQUxTRSkKICAgIGFibGluZShhID0gMCwKICAgICAgICAgICBiID0gMSwKICAgICAgICAgICBjb2wgPSAncmVkJywKICAgICAgICAgICBsdHkgPSAyKQogICAgdGV4dCgwLjgsCiAgICAgICAgIDAuMjUsCiAgICAgICAgIHBhc3RlKCdBVVJPQzonLCBzcHJpbnRmKCclLjNmJywgcm91bmQoYXVyb2MsIDMpKSkpCn0KYGBgCgoKYGBge3J9CmNvbXB1dGVfYXVwciA8LSBmdW5jdGlvbihwcmVkaWN0aW9ucywgbGFiZWxzLCB0aXRsZSkgewogICAgbl9ibGFjayA8LSBsZW5ndGgobGFiZWxzW2xhYmVscyA9PSAxXSkKICAgIG5fdG90YWwgPC0gbGVuZ3RoKGxhYmVscykKICAgIHByZWQgPC0gcHJlZGljdGlvbihwcmVkaWN0aW9ucywgbGFiZWxzKQogICAgYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKICAgIGF1cHIgPC0gYXVwckB5LnZhbHVlc1tbMV1dCiAgICBwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsICdwcmVjJywgJ3JlYycpCiAgICBwbG90KHBlcmYsCiAgICAgICAgIG1haW4gPSB0aXRsZSwKICAgICAgICAgeGxpbT1jKDAsMSksCiAgICAgICAgIHlsaW09YygwLDEpLAogICAgICAgICBjb2xvcml6ZSA9IEZBTFNFKQogICAgYWJsaW5lKGEgPSBuX2JsYWNrL25fdG90YWwsCiAgICAgICAgICAgYiA9IDAsCiAgICAgICAgICAgY29sID0gJ3JlZCcsCiAgICAgICAgICAgbHR5ID0gMikKICAgIHRleHQoMC4yLAogICAgICAgICAwLjIsCiAgICAgICAgIHBhc3RlKCdOdWxsIEFVUFI6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKG5fYmxhY2sgLyBuX3RvdGFsLCAzKSkpKQogICAgdGV4dCgwLjgsCiAgICAgICAgIDAuMiwKICAgICAgICAgcGFzdGUoJ0FVUFI6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKGF1cHIsIDMpKSkpCn0KYGBgCgoKYGBge3J9CmNvbmZ1c2lvbl9tYXRyaXggPC0gZnVuY3Rpb24oZGYsIGNvbHVtbiwgdGhyZXNob2xkPTAuNSkgewogICAgcHJlZHMgPC0gYXMuZmFjdG9yKGlmX2Vsc2UoZGZbY29sdW1uXSA+PSB0aHJlc2hvbGQsIDEsIDApKQogICAgY29uZnVzaW9uTWF0cml4KHByZWRzLCBkZiRsYWJlbCwgcG9zaXRpdmUgPSAnMScpCn0KYGBgCgoKCiMgTG9hZCBkYXRhCmBgYHtyfQppcm9wX2RhdGEgPC0gcmVhZF9jc3YoJy9Wb2x1bWVzL0V4dGVybmFsL2lyb3BfZGF0YS9pcm9wXzA3MDkyMDIwLmNzdicpICU+JQogICAgY2xlYW5fbmFtZXMoKSAlPiUKICAgIGRpc3RpbmN0KHN1YmplY3RfaWQsIC5rZWVwX2FsbCA9IFRSVUUpICU+JQogICAgc2VsZWN0KHN1YmplY3RfaWQsIHJhY2UpCgp0ZXN0X2RhdGEgPC0gcmVhZF9jc3YoJy4vb3V0L2RhdGFzZXRzL3Rlc3RfZGF0YS5jc3YnKSAlPiUKICAgIHNlbGVjdChzdWJqZWN0X2lkLCBpbWFnZV9pZCkKCgppbWFnZV9sZXZlbCA8LSByZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9yZXRjYW1fZmlsdGVyZWRfMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpICU+JQogICAgbXV0YXRlKGxhYmVsID0gYXMuZmFjdG9yKGlmX2Vsc2Uoc3RyX2RldGVjdChpbWdfbG9jLCAnYmxhY2snKSwgMSwgMCkpLAogICAgICAgICAgIGltYWdlX2lkID0gYXMubnVtZXJpYyhmaWxlX3BhdGhfc2Fuc19leHQoYmFzZW5hbWUoaW1nX2xvYykpKSkgJT4lCiAgICBzZWxlY3QoaW1nX2xvYywgaW1hZ2VfaWQsIGxhYmVsLCByZXRjYW0gPSBwcm9iYWJpbGl0eSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3JldGNhbV9maWx0ZXJlZF8wX3JhbmRvbS5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCByZXRjYW1fcmFuZG9tID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIAogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfcmFuZG9tLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8wX3JhbmRvbSA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8wX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzUwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF81MF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzUwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl81MF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xMDAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTAwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTAwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTAwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzE1MC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTUwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xNTBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xNTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xNTBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjAwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMDAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIwMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIwMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIwMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIxMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjEwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjEwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjEwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIyMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjIwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMjBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMjBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMjBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMjBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjMwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMzAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIzMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIzMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIzMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNDAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI0MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjQwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjQwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjQwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1MC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjUwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjU3LmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTcgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1N19iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI1N19iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1N19za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI1N19za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8wXzEwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF8xMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF8xMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzc1XzE1MC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfNzUgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzc1XzE1MF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzc1X2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNzVfMTUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfNzVfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIAogICAgaW5uZXJfam9pbih0ZXN0X2RhdGEsIGJ5ID0gJ2ltYWdlX2lkJykgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgZXZlcnl0aGluZygpLCAtaW1nX2xvYykKCgoKc3ViamVjdF9sZXZlbCA8LSBpbWFnZV9sZXZlbCAlPiUKICAgIGdyb3VwX2J5KHN1YmplY3RfaWQpICU+JQogICAgbXV0YXRlKGFjcm9zcyhjKC1pbWFnZV9pZCwgLWxhYmVsKSwgbWVkaWFuKSkgJT4lCiAgICB1bmdyb3VwKCkgJT4lCiAgICBkaXN0aW5jdChzdWJqZWN0X2lkLCAua2VlcF9hbGwgPSBUUlVFKQpgYGAKCgojIEltYWdlLWxldmVsIEFuYWx5c2lzCgojIyMgUmV0Q2FtIEltYWdlcwoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRyZXRjYW0sIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRyZXRjYW0sIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBSYXcgUmV0Q2FtIEltYWdlcycpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdyZXRjYW0nKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkcmV0Y2FtX3JhbmRvbSwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogUmF3IFJldENhbSBJbWFnZXMgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJHJldGNhbV9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBSYXcgUmV0Q2FtIEltYWdlcyAtIFNodWZmbGVkIExhYmVscycpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdyZXRjYW1fcmFuZG9tJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8wLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX3NrZWxldG9uaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8wX3JhbmRvbSwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzBfcmFuZG9tJykKYGBgCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl81MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl81MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl81MCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzUwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTAwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTUwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTUwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjAwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjAwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjEwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjEwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjIwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMjAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMjBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjIwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMjBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjMwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjMwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMzBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjQwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjUwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjUwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1NywgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3JykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjU3LCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3JykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTcnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1N19iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NyBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTdfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjU3X2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1N19za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTdfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPiAxMAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNzUsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1LCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl83NScpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNzVfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl83NV9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl83NV9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl83NV9za2VsZXRvbml6ZWQnKQpgYGAKCgoKCgoKCgoKCgojIFN1YmplY3QtbGV2ZWwgQW5hbHlzaXMKCiMjIyBSZXRDYW0gSW1hZ2VzCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkcmV0Y2FtLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJHJldGNhbSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogUmF3IFJldENhbSBJbWFnZXMnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdyZXRjYW0nKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRyZXRjYW1fcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJHJldGNhbV9yYW5kb20sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFJhdyBSZXRDYW0gSW1hZ2VzIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAncmV0Y2FtX3JhbmRvbScpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8wLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgc2tlbGV0b25pemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMF9za2VsZXRvbml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCAtIFNodWZmbGVkIExhYmVscycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMF9yYW5kb20nKQpgYGAKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl81MCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl81MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzE1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIxMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMjAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIyMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIzMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI0MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NwoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTcsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3JykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTcsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTcnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTdfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1N19iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTdfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1N19za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA+IDEwCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzUsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl83NScpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl83NV9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzc1X2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl83NV9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzc1X3NrZWxldG9uaXplZCcpCmBgYAoKCgpgYGB7cn0KYXVwcl9maWd1cmVzIDwtIGZ1bmN0aW9uKHByZWRpY3Rpb25zLCBsYWJlbHMsIHRpdGxlLCBuYW1lcywgc2F2ZV9uYW1lKSB7CiAgICBuX2JsYWNrIDwtIGxlbmd0aChsYWJlbHNbbGFiZWxzID09IDFdKQogICAgbl90b3RhbCA8LSBsZW5ndGgobGFiZWxzKQogICAgZm9yIChpIGluIHNlcV9hbG9uZyhwcmVkaWN0aW9ucykpIHsKICAgICAgICBwcmVkcyA8LSBwcmVkaWN0aW9uKHByZWRpY3Rpb25zW2ldLCBsYWJlbHMpCiAgICAgICAgYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkcywgbWVhc3VyZSA9ICdhdWNwcicpCiAgICAgICAgYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KICAgICAgICBwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWRzLCAncHJlYycsICdyZWMnKQogICAgICAgIGlmIChpcy5udWxsKGRldi5saXN0KCkpKSB7CiAgICAgICAgICAgIHBuZyhzYXZlX25hbWUsCiAgICAgICAgICAgICAgICB3aWR0aD03LAogICAgICAgICAgICAgICAgaGVpZ2h0PTUsCiAgICAgICAgICAgICAgICB1bml0cz0naW4nLAogICAgICAgICAgICAgICAgcmVzPTMwMCkKICAgICAgICAgICAgcGxvdChwZXJmLAogICAgICAgICAgICAgICAgIG1haW4gPSB0aXRsZSwKICAgICAgICAgICAgICAgICB4bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICBjb2wgPSBpLAogICAgICAgICAgICAgICAgIGx3ZCA9IDIpCiAgICAgICAgfSBlbHNlIHsKICAgICAgICAgICAgcGxvdChwZXJmLAogICAgICAgICAgICAgICAgIG1haW4gPSB0aXRsZSwKICAgICAgICAgICAgICAgICB4bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICBjb2wgPSBpLAogICAgICAgICAgICAgICAgIGx3ZCA9IDIsCiAgICAgICAgICAgICAgICAgYWRkID0gVFJVRSkKICAgICAgICB9CiAgICB9CiAgICBhYmxpbmUoYSA9IG5fYmxhY2svbl90b3RhbCwKICAgICAgICAgICBiID0gMCwKICAgICAgICAgICBjb2wgPSAncmVkJywKICAgICAgICAgICBsdHkgPSAyLAogICAgICAgICAgIGx3ZCA9IDIpCiAgICBsZWdlbmQoJ2JvdHRvbWxlZnQnLAogICAgICAgICAgIG5hbWVzLAogICAgICAgICAgIGx3ZCA9IDIsCiAgICAgICAgICAgY29sID0gc2VxX2Fsb25nKHByZWRpY3Rpb25zKSwKICAgICAgICAgICBidHkgPSAnbicsCiAgICAgICAgICAgaW5zZXQgPSBjKDAuMSwgMC4wNykpCiAgICBsZWdlbmQoJ2JvdHRvbWxlZnQnLAogICAgICAgICAgIHBhc3RlKCdOdWxsIEFVUFI6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKG5fYmxhY2sgLyBuX3RvdGFsLCAzKSkpLAogICAgICAgICAgIGx3ZCA9IDIsCiAgICAgICAgICAgbHR5ID0gMiwKICAgICAgICAgICBjb2wgPSAncmVkJywKICAgICAgICAgICBidHkgPSAnbicsCiAgICAgICAgICAgaW5zZXQgPSBjKDAuMSwgMC4wKSkKfQpgYGAKCgojIyBGaWd1cmVzIGZvciBQYXBlcgpgYGB7cn0KYXVwcl9maWd1cmVzKGxpc3QoaW1hZ2VfbGV2ZWwkZmlsdGVyXzAsIGltYWdlX2xldmVsJGZpbHRlcl81MCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MCksCiAgICAgICAgICAgICBpbWFnZV9sZXZlbCRsYWJlbCwKICAgICAgICAgICAgICdQcmVjaXNpb24tUmVjYWxsIEN1cnZlcyBvZiBUaHJlc2hvbGRlZCBSVk1zJywKICAgICAgICAgICAgIGMoJ1RocmVzaG9sZCAwIEFVUFI6JywgJ1RocmVzaG9sZCA1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDE1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDIyMCBBVVBSOiAnKSwKICAgICAgICAgICAgICcuL291dC9maWd1cmVzL3RocmVzaG9sZGVkX2ltYWdlLnBuZycpCmBgYAoKYGBge3J9CmF1cHJfZmlndXJlcyhsaXN0KGltYWdlX2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkKSwKICAgICAgICAgICAgIGltYWdlX2xldmVsJGxhYmVsLAogICAgICAgICAgICAgJ1ByZWNpc2lvbi1SZWNhbGwgQ3VydmVzIG9mIEJpbmFyaXplZCBSVk1zJywKICAgICAgICAgICAgIGMoJ1RocmVzaG9sZCAwIEFVUFI6JywgJ1RocmVzaG9sZCA1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDE1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDIyMCBBVVBSOiAnKSwKICAgICAgICAgICAgICcuL291dC9maWd1cmVzL2JpbmFyaXplZF9pbWFnZS5wbmcnKQpgYGAKCmBgYHtyfQphdXByX2ZpZ3VyZXMobGlzdChpbWFnZV9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCksCiAgICAgICAgICAgICBpbWFnZV9sZXZlbCRsYWJlbCwKICAgICAgICAgICAgICdQcmVjaXNpb24tUmVjYWxsIEN1cnZlcyBvZiBTa2VsZXRvbml6ZWQgUlZNcycsCiAgICAgICAgICAgICBjKCdUaHJlc2hvbGQgMCBBVVBSOicsICdUaHJlc2hvbGQgNTAgQVVQUjogJywgJ1RocmVzaG9sZCAxNTAgQVVQUjogJywgJ1RocmVzaG9sZCAyMjAgQVVQUjogJyksCiAgICAgICAgICAgICAnLi9vdXQvZmlndXJlcy9za2VsZXRvbml6ZWRfaW1hZ2UucG5nJykKYGBgCgpgYGB7cn0KYXVwcl9maWd1cmVzKGxpc3Qoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMCwgc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTAsIHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMCwgc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwKSwKICAgICAgICAgICAgIHN1YmplY3RfbGV2ZWwkbGFiZWwsCiAgICAgICAgICAgICAnUHJlY2lzaW9uLVJlY2FsbCBDdXJ2ZXMgb2YgVGhyZXNob2xkZWQgUlZNcycsCiAgICAgICAgICAgICBjKCdUaHJlc2hvbGQgMCBBVVBSOicsICdUaHJlc2hvbGQgNTAgQVVQUjogJywgJ1RocmVzaG9sZCAxNTAgQVVQUjogJywgJ1RocmVzaG9sZCAyMjAgQVVQUjogJyksCiAgICAgICAgICAgICAnLi9vdXQvZmlndXJlcy90aHJlc2hvbGRlZF9zdWJqZWN0LnBuZycpCmBgYAoKYGBge3J9CmF1cHJfZmlndXJlcyhsaXN0KHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI0MF9iaW5hcml6ZWQpLAogICAgICAgICAgICAgc3ViamVjdF9sZXZlbCRsYWJlbCwKICAgICAgICAgICAgICdQcmVjaXNpb24tUmVjYWxsIEN1cnZlcyBvZiBCaW5hcml6ZWQgUlZNcycsCiAgICAgICAgICAgICBjKCdUaHJlc2hvbGQgMCBBVVBSOicsICdUaHJlc2hvbGQgNTAgQVVQUjogJywgJ1RocmVzaG9sZCAxNTAgQVVQUjogJywgJ1RocmVzaG9sZCAyMjAgQVVQUjogJyksCiAgICAgICAgICAgICAnLi9vdXQvZmlndXJlcy9iaW5hcml6ZWRfc3ViamVjdC5wbmcnKQpgYGAKCmBgYHtyfQphdXByX2ZpZ3VyZXMobGlzdChzdWJqZWN0X2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDBfc2tlbGV0b25pemVkKSwKICAgICAgICAgICAgIHN1YmplY3RfbGV2ZWwkbGFiZWwsCiAgICAgICAgICAgICAnUHJlY2lzaW9uLVJlY2FsbCBDdXJ2ZXMgb2YgU2tlbGV0b25pemVkIFJWTXMnLAogICAgICAgICAgICAgYygnVGhyZXNob2xkIDAgQVVQUjonLCAnVGhyZXNob2xkIDUwIEFVUFI6ICcsICdUaHJlc2hvbGQgMTUwIEFVUFI6ICcsICdUaHJlc2hvbGQgMjIwIEFVUFI6ICcpLAogICAgICAgICAgICAgJy4vb3V0L2ZpZ3VyZXMvc2tlbGV0b25pemVkX3N1YmplY3QucG5nJykKYGBgCgoKCiMjIEV2YWx1YXRlIG9uIE1hbnVhbCBTZWdtZW50YXRpb25zCmBgYHtyfQptYW51YWxfYmluYXJ5XzAgPC0gcmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvbWFudWFsX3NlZ21lbnRhdGlvbnNfYmluYXJpemVkXzAuY3N2JykgJT4lCiAgICBtdXRhdGUoaW1nX25hbWUgPSBiYXNlbmFtZShpbWdfbG9jKSkgJT4lCiAgICBzZXBhcmF0ZShpbWdfbmFtZSwgYygnc2l0ZScsICdzdWJqZWN0X2lkJykpICU+JQogICAgZmlsdGVyKCFpcy5uYShhcy5udW1lcmljKHN1YmplY3RfaWQpKSkgJT4lCiAgICBtdXRhdGUoc3ViamVjdF9pZCA9IHBhc3RlKHRvdXBwZXIoc2l0ZSksIHN1YmplY3RfaWQsIHNlcCA9ICctJykpICU+JQogICAgc2VsZWN0KHN1YmplY3RfaWQsIHByb2JhYmlsaXR5KSAlPiUKICAgIGlubmVyX2pvaW4oaXJvcF9kYXRhKSAlPiUKICAgIGZpbHRlcihyYWNlID09ICdBZnJpY2FuIEFtZXJpY2FuJyB8IHJhY2UgPT0gJ0NhdWNhc2lhbi9XaGl0ZScpICU+JQogICAgbXV0YXRlKHJhY2UgPSBpZl9lbHNlKHJhY2UgPT0gJ0FmcmljYW4gQW1lcmljYW4nLCAxLCAwKSwKICAgICAgICAgICBhY3Jvc3MocmFjZSwgYXMuZmFjdG9yKSkKCm1hbnVhbF9za2VsZXRvbl8wIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL21hbnVhbF9zZWdtZW50YXRpb25zX3NrZWxldG9uaXplZF8wLmNzdicpICU+JQogICAgbXV0YXRlKGltZ19uYW1lID0gYmFzZW5hbWUoaW1nX2xvYykpICU+JQogICAgc2VwYXJhdGUoaW1nX25hbWUsIGMoJ3NpdGUnLCAnc3ViamVjdF9pZCcpKSAlPiUKICAgIGZpbHRlcighaXMubmEoYXMubnVtZXJpYyhzdWJqZWN0X2lkKSkpICU+JQogICAgbXV0YXRlKHN1YmplY3RfaWQgPSBwYXN0ZSh0b3VwcGVyKHNpdGUpLCBzdWJqZWN0X2lkLCBzZXAgPSAnLScpKSAlPiUKICAgIHNlbGVjdChzdWJqZWN0X2lkLCBwcm9iYWJpbGl0eSkgJT4lCiAgICBpbm5lcl9qb2luKGlyb3BfZGF0YSkgJT4lCiAgICBmaWx0ZXIocmFjZSA9PSAnQWZyaWNhbiBBbWVyaWNhbicgfCByYWNlID09ICdDYXVjYXNpYW4vV2hpdGUnKSAlPiUKICAgIG11dGF0ZShyYWNlID0gaWZfZWxzZShyYWNlID09ICdBZnJpY2FuIEFtZXJpY2FuJywgMSwgMCksCiAgICAgICAgICAgYWNyb3NzKHJhY2UsIGFzLmZhY3RvcikpCgptYW51YWxfYmluYXJ5XzUwIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL21hbnVhbF9zZWdtZW50YXRpb25zX2JpbmFyaXplZF81MC5jc3YnKSAlPiUKICAgIG11dGF0ZShpbWdfbmFtZSA9IGJhc2VuYW1lKGltZ19sb2MpKSAlPiUKICAgIHNlcGFyYXRlKGltZ19uYW1lLCBjKCdzaXRlJywgJ3N1YmplY3RfaWQnKSkgJT4lCiAgICBmaWx0ZXIoIWlzLm5hKGFzLm51bWVyaWMoc3ViamVjdF9pZCkpKSAlPiUKICAgIG11dGF0ZShzdWJqZWN0X2lkID0gcGFzdGUodG91cHBlcihzaXRlKSwgc3ViamVjdF9pZCwgc2VwID0gJy0nKSkgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgcHJvYmFiaWxpdHkpICU+JQogICAgaW5uZXJfam9pbihpcm9wX2RhdGEpICU+JQogICAgZmlsdGVyKHJhY2UgPT0gJ0FmcmljYW4gQW1lcmljYW4nIHwgcmFjZSA9PSAnQ2F1Y2FzaWFuL1doaXRlJykgJT4lCiAgICBtdXRhdGUocmFjZSA9IGlmX2Vsc2UocmFjZSA9PSAnQWZyaWNhbiBBbWVyaWNhbicsIDEsIDApLAogICAgICAgICAgIGFjcm9zcyhyYWNlLCBhcy5mYWN0b3IpKQoKbWFudWFsX3NrZWxldG9uXzUwIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL21hbnVhbF9zZWdtZW50YXRpb25zX3NrZWxldG9uaXplZF81MC5jc3YnKSAlPiUKICAgIG11dGF0ZShpbWdfbmFtZSA9IGJhc2VuYW1lKGltZ19sb2MpKSAlPiUKICAgIHNlcGFyYXRlKGltZ19uYW1lLCBjKCdzaXRlJywgJ3N1YmplY3RfaWQnKSkgJT4lCiAgICBmaWx0ZXIoIWlzLm5hKGFzLm51bWVyaWMoc3ViamVjdF9pZCkpKSAlPiUKICAgIG11dGF0ZShzdWJqZWN0X2lkID0gcGFzdGUodG91cHBlcihzaXRlKSwgc3ViamVjdF9pZCwgc2VwID0gJy0nKSkgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgcHJvYmFiaWxpdHkpICU+JQogICAgaW5uZXJfam9pbihpcm9wX2RhdGEpICU+JQogICAgZmlsdGVyKHJhY2UgPT0gJ0FmcmljYW4gQW1lcmljYW4nIHwgcmFjZSA9PSAnQ2F1Y2FzaWFuL1doaXRlJykgJT4lCiAgICBtdXRhdGUocmFjZSA9IGlmX2Vsc2UocmFjZSA9PSAnQWZyaWNhbiBBbWVyaWNhbicsIDEsIDApLAogICAgICAgICAgIGFjcm9zcyhyYWNlLCBhcy5mYWN0b3IpKQpgYGAKCgpgYGB7cn0KcHJlZCA8LSBwcmVkaWN0aW9uKG1hbnVhbF9iaW5hcnlfMCRwcm9iYWJpbGl0eSwgbWFudWFsX2JpbmFyeV8wJHJhY2UpCgpudW1feWVzIDwtIGxlbmd0aChtYW51YWxfYmluYXJ5XzAkcmFjZVttYW51YWxfYmluYXJ5XzAkcmFjZSA9PSAxXSkKbnVtX25vIDwtIGxlbmd0aChtYW51YWxfYmluYXJ5XzAkcmFjZVttYW51YWxfYmluYXJ5XzAkcmFjZSA9PSAwXSkKCnBlcmYgPC0gcGVyZm9ybWFuY2UocHJlZCwncHJlYycsICdyZWMnKQpwbG90KHBlcmYsIGNvbG9yaXplPVRSVUUsIG1haW4gPSAnTWFudWFsIFNlZ21lbnRhdGlvbnM6IEJpbmFyaXplZCAtIDAgVGhyZXNob2xkIE1vZGVsJywgeWxpbSA9IGMoMCwxKSkKYWJsaW5lKGEgPSBudW1feWVzIC8gbnVtX25vLCBiID0gMCwgY29sID0gJ3JlZCcsIGx0eSA9IDIpCmF1cHIgPC0gcGVyZm9ybWFuY2UocHJlZCwgbWVhc3VyZSA9ICdhdWNwcicpCmF1cHIgPC0gYXVwckB5LnZhbHVlc1tbMV1dCnRleHQoMC44LCAwLjksIHBhc3RlKCdBVVBSOiAnLCBzcHJpbnRmKCclLjNmJywgcm91bmQoYXVwciwgMykpLCBzZXAgPSAnJykpCmBgYAoKCmBgYHtyfQpwcmVkIDwtIHByZWRpY3Rpb24obWFudWFsX3NrZWxldG9uXzAkcHJvYmFiaWxpdHksIG1hbnVhbF9za2VsZXRvbl8wJHJhY2UpCgpudW1feWVzIDwtIGxlbmd0aChtYW51YWxfc2tlbGV0b25fMCRyYWNlW21hbnVhbF9za2VsZXRvbl8wJHJhY2UgPT0gMV0pCm51bV9ubyA8LSBsZW5ndGgobWFudWFsX3NrZWxldG9uXzAkcmFjZVttYW51YWxfc2tlbGV0b25fMCRyYWNlID09IDBdKQoKcGVyZiA8LSBwZXJmb3JtYW5jZShwcmVkLCdwcmVjJywgJ3JlYycpCnBsb3QocGVyZiwgY29sb3JpemU9VFJVRSwgbWFpbiA9ICdNYW51YWwgU2VnbWVudGF0aW9uczogU2tlbGV0b25pemVkIC0gMCBUaHJlc2hvbGQgTW9kZWwnLCB5bGltID0gYygwLDEpKQphYmxpbmUoYSA9IG51bV95ZXMgLyBudW1fbm8sIGIgPSAwLCBjb2wgPSAncmVkJywgbHR5ID0gMikKYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KdGV4dCgwLjgsIDAuOSwgcGFzdGUoJ0FVUFI6ICcsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSksIHNlcCA9ICcnKSkKYGBgCgoKYGBge3J9CnByZWQgPC0gcHJlZGljdGlvbihtYW51YWxfYmluYXJ5XzUwJHByb2JhYmlsaXR5LCBtYW51YWxfYmluYXJ5XzUwJHJhY2UpCgpudW1feWVzIDwtIGxlbmd0aChtYW51YWxfYmluYXJ5XzUwJHJhY2VbbWFudWFsX2JpbmFyeV81MCRyYWNlID09IDFdKQpudW1fbm8gPC0gbGVuZ3RoKG1hbnVhbF9iaW5hcnlfNTAkcmFjZVttYW51YWxfYmluYXJ5XzUwJHJhY2UgPT0gMF0pCgpwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsJ3ByZWMnLCAncmVjJykKcGxvdChwZXJmLCBjb2xvcml6ZT1UUlVFLCBtYWluID0gJ01hbnVhbCBTZWdtZW50YXRpb25zOiBCaW5hcml6ZWQgLSA1MCBUaHJlc2hvbGQgTW9kZWwnLCB5bGltID0gYygwLDEpKQphYmxpbmUoYSA9IG51bV95ZXMgLyBudW1fbm8sIGIgPSAwLCBjb2wgPSAncmVkJywgbHR5ID0gMikKYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KdGV4dCgwLjgsIDAuOSwgcGFzdGUoJ0FVUFI6ICcsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSksIHNlcCA9ICcnKSkKYGBgCgoKYGBge3J9CnByZWQgPC0gcHJlZGljdGlvbihtYW51YWxfc2tlbGV0b25fNTAkcHJvYmFiaWxpdHksIG1hbnVhbF9za2VsZXRvbl81MCRyYWNlKQoKbnVtX3llcyA8LSBsZW5ndGgobWFudWFsX3NrZWxldG9uXzUwJHJhY2VbbWFudWFsX3NrZWxldG9uXzUwJHJhY2UgPT0gMV0pCm51bV9ubyA8LSBsZW5ndGgobWFudWFsX3NrZWxldG9uXzUwJHJhY2VbbWFudWFsX3NrZWxldG9uXzUwJHJhY2UgPT0gMF0pCgpwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsJ3ByZWMnLCAncmVjJykKcGxvdChwZXJmLCBjb2xvcml6ZT1UUlVFLCBtYWluID0gJ01hbnVhbCBTZWdtZW50YXRpb25zOiBTa2VsZXRvbml6ZWQgLSA1MCBUaHJlc2hvbGQgTW9kZWwnLCB5bGltID0gYygwLDEpKQphYmxpbmUoYSA9IG51bV95ZXMgLyBudW1fbm8sIGIgPSAwLCBjb2wgPSAncmVkJywgbHR5ID0gMikKYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KdGV4dCgwLjgsIDAuOSwgcGFzdGUoJ0FVUFI6ICcsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSksIHNlcCA9ICcnKSkKYGBgCgoKYGBge3J9CmltYWdlX2xldmVsIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3JldGNhbV9maWx0ZXJlZF8wLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSkgJT4lCiAgICBtdXRhdGUobGFiZWwgPSBhcy5mYWN0b3IoaWZfZWxzZShzdHJfZGV0ZWN0KGltZ19sb2MsICdibGFjaycpLCAxLCAwKSksCiAgICAgICAgICAgaW1hZ2VfaWQgPSBhcy5udW1lcmljKGZpbGVfcGF0aF9zYW5zX2V4dChiYXNlbmFtZShpbWdfbG9jKSkpKSAlPiUKICAgIHNlbGVjdChpbWdfbG9jLCBpbWFnZV9pZCwgbGFiZWwsIHJldGNhbSA9IHByb2JhYmlsaXR5KSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19iaW5hcml6ZWRfdGVzdC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpCmBgYAoKYGBge3J9CnByZWQgPC0gcHJlZGljdGlvbihpbWFnZV9sZXZlbCRmaWx0ZXJfMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsKQoKbnVtX3llcyA8LSBsZW5ndGgoaW1hZ2VfbGV2ZWwkbGFiZWxbaW1hZ2VfbGV2ZWwkbGFiZWwgPT0gMV0pCm51bV9ubyA8LSBsZW5ndGgoaW1hZ2VfbGV2ZWwkbGFiZWxbaW1hZ2VfbGV2ZWwkbGFiZWwgPT0gMF0pCgpwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsJ3ByZWMnLCAncmVjJykKcGxvdChwZXJmLCBjb2xvcml6ZT1UUlVFLCBtYWluID0gJ01hbnVhbCBTZWdtZW50YXRpb25zOiBTa2VsZXRvbml6ZWQgLSAwIFRocmVzaG9sZCBNb2RlbCcsIHlsaW0gPSBjKDAsMSkpCmFibGluZShhID0gbnVtX3llcyAvIG51bV9ubywgYiA9IDAsIGNvbCA9ICdyZWQnLCBsdHkgPSAyKQphdXByIDwtIHBlcmZvcm1hbmNlKHByZWQsIG1lYXN1cmUgPSAnYXVjcHInKQphdXByIDwtIGF1cHJAeS52YWx1ZXNbWzFdXQp0ZXh0KDAuOCwgMC45LCBwYXN0ZSgnQVVQUjogJywgc3ByaW50ZignJS4zZicsIHJvdW5kKGF1cHIsIDMpKSwgc2VwID0gJycpKQpgYGA=